<html>

<head>
<meta http-equiv=Content-Type content="text/html; charset=gb2312">
<meta name=Generator content="Microsoft Word 12 (filtered)">
<style>
<!--
 /* Font Definitions */
 @font-face
	{font-family:宋体;
	panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
	{font-family:"Cambria Math";
	panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
	{font-family:Cambria;
	panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
	{font-family:Calibri;
	panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
	{font-family:Consolas;
	panose-1:2 11 6 9 2 2 4 3 2 4;}
@font-face
	{font-family:"\@宋体";
	panose-1:2 1 6 0 3 1 1 1 1 1;}
 /* Style Definitions */
 p.MsoNormal, li.MsoNormal, div.MsoNormal
	{margin:0cm;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
h1
	{mso-style-link:"标题 1 Char";
	margin-top:17.0pt;
	margin-right:0cm;
	margin-bottom:16.5pt;
	margin-left:0cm;
	text-align:justify;
	text-justify:inter-ideograph;
	line-height:240%;
	page-break-after:avoid;
	font-size:22.0pt;
	font-family:"Calibri","sans-serif";}
h2
	{mso-style-link:"标题 2 Char";
	margin-top:13.0pt;
	margin-right:0cm;
	margin-bottom:13.0pt;
	margin-left:21.25pt;
	text-align:justify;
	text-justify:inter-ideograph;
	text-indent:-21.25pt;
	line-height:173%;
	page-break-after:avoid;
	font-size:16.0pt;
	font-family:"Cambria","serif";}
h3
	{mso-style-link:"标题 3 Char";
	margin-top:13.0pt;
	margin-right:0cm;
	margin-bottom:13.0pt;
	margin-left:49.6pt;
	text-align:justify;
	text-justify:inter-ideograph;
	text-indent:-1.0cm;
	line-height:173%;
	page-break-after:avoid;
	font-size:16.0pt;
	font-family:"Calibri","sans-serif";}
h4
	{mso-style-link:"标题 4 Char";
	margin-top:14.0pt;
	margin-right:0cm;
	margin-bottom:14.5pt;
	margin-left:70.9pt;
	text-align:justify;
	text-justify:inter-ideograph;
	text-indent:-1.0cm;
	line-height:156%;
	page-break-after:avoid;
	font-size:14.0pt;
	font-family:"Cambria","serif";}
h5
	{mso-style-link:"标题 5 Char";
	margin-top:14.0pt;
	margin-right:0cm;
	margin-bottom:14.5pt;
	margin-left:99.2pt;
	text-align:justify;
	text-justify:inter-ideograph;
	text-indent:-35.4pt;
	line-height:156%;
	page-break-after:avoid;
	font-size:14.0pt;
	font-family:"Calibri","sans-serif";}
p.MsoToc2, li.MsoToc2, div.MsoToc2
	{margin-top:0cm;
	margin-right:0cm;
	margin-bottom:0cm;
	margin-left:21.0pt;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
p.MsoToc3, li.MsoToc3, div.MsoToc3
	{margin-top:0cm;
	margin-right:0cm;
	margin-bottom:0cm;
	margin-left:42.0pt;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
p.MsoToc4, li.MsoToc4, div.MsoToc4
	{margin-top:0cm;
	margin-right:0cm;
	margin-bottom:0cm;
	margin-left:63.0pt;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
p.MsoToc5, li.MsoToc5, div.MsoToc5
	{margin-top:0cm;
	margin-right:0cm;
	margin-bottom:0cm;
	margin-left:84.0pt;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
p.MsoHeader, li.MsoHeader, div.MsoHeader
	{mso-style-link:"页眉 Char";
	margin:0cm;
	margin-bottom:.0001pt;
	text-align:center;
	layout-grid-mode:char;
	border:none;
	padding:0cm;
	font-size:9.0pt;
	font-family:"Calibri","sans-serif";}
p.MsoFooter, li.MsoFooter, div.MsoFooter
	{mso-style-link:"页脚 Char";
	margin:0cm;
	margin-bottom:.0001pt;
	layout-grid-mode:char;
	font-size:9.0pt;
	font-family:"Calibri","sans-serif";}
a:link, span.MsoHyperlink
	{color:blue;
	text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
	{color:purple;
	text-decoration:underline;}
code
	{font-family:宋体;}
pre
	{mso-style-link:"HTML 预设格式 Char";
	margin-top:7.5pt;
	margin-right:0cm;
	margin-bottom:7.5pt;
	margin-left:0cm;
	font-size:12.0pt;
	font-family:宋体;}
p.MsoAcetate, li.MsoAcetate, div.MsoAcetate
	{mso-style-link:"批注框文本 Char";
	margin:0cm;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	font-size:9.0pt;
	font-family:"Calibri","sans-serif";}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
	{margin:0cm;
	margin-bottom:.0001pt;
	text-align:justify;
	text-justify:inter-ideograph;
	text-indent:21.0pt;
	font-size:10.5pt;
	font-family:"Calibri","sans-serif";}
p.MsoTocHeading, li.MsoTocHeading, div.MsoTocHeading
	{margin-top:24.0pt;
	margin-right:0cm;
	margin-bottom:0cm;
	margin-left:0cm;
	margin-bottom:.0001pt;
	line-height:115%;
	page-break-after:avoid;
	font-size:14.0pt;
	font-family:"Cambria","serif";
	color:#365F91;
	font-weight:bold;}
span.Char
	{mso-style-name:"页眉 Char";
	mso-style-link:页眉;}
span.Char0
	{mso-style-name:"页脚 Char";
	mso-style-link:页脚;}
span.HTMLChar
	{mso-style-name:"HTML 预设格式 Char";
	mso-style-link:"HTML 预设格式";
	font-family:宋体;}
span.Char1
	{mso-style-name:"批注框文本 Char";
	mso-style-link:批注框文本;}
span.2Char
	{mso-style-name:"标题 2 Char";
	mso-style-link:"标题 2";
	font-family:"Cambria","serif";
	font-weight:bold;}
span.3Char
	{mso-style-name:"标题 3 Char";
	mso-style-link:"标题 3";
	font-weight:bold;}
span.4Char
	{mso-style-name:"标题 4 Char";
	mso-style-link:"标题 4";
	font-family:"Cambria","serif";
	font-weight:bold;}
span.5Char
	{mso-style-name:"标题 5 Char";
	mso-style-link:"标题 5";
	font-weight:bold;}
span.1Char
	{mso-style-name:"标题 1 Char";
	mso-style-link:"标题 1";
	font-weight:bold;}
 /* Page Definitions */
 @page Section1
	{size:595.3pt 841.9pt;
	margin:72.0pt 90.0pt 72.0pt 90.0pt;
	layout-grid:15.6pt;}
div.Section1
	{page:Section1;}
 /* List Definitions */
 ol
	{margin-bottom:0cm;}
ul
	{margin-bottom:0cm;}
-->
</style>

</head>

<body lang=ZH-CN link=blue vlink=purple style='text-justify-trim:punctuation'>

<div class=Section1 style='layout-grid:15.6pt'>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><span style='font-size:16.0pt;font-family:宋体'>林氏物语</span><span
lang=EN-US style='font-size:16.0pt'>.</span><span style='font-size:16.0pt;
font-family:宋体'>技术乱弹之</span><span lang=EN-US style='font-size:16.0pt'>hive</span><span
style='font-size:16.0pt;font-family:宋体'>源码之词法语法解析</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp; </span></p>

<p class=MsoToc2><span
lang=EN-US><a href="#_Toc368923110"><span lang=EN-US style='font-family:宋体'><span
lang=EN-US>第一章</span></span><span style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>前言</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>1</span></a></span></p>

<p class=MsoToc2><span lang=EN-US><a href="#_Toc368923111"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第二章</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>准备工作</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>2</span></a></span></p>

<p class=MsoToc2><span lang=EN-US><a href="#_Toc368923112"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第三章</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>总体概览</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>4</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923113"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第一节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span>hive<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>官方架构图</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>4</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923114"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第二节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>流程处理图</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>4</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923115"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第三节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>具体框架图</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>6</span></a></span></p>

<p class=MsoToc2><span lang=EN-US><a href="#_Toc368923116"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第四章</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>语法分析</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>6</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923117"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第一节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>快速定位词法语法分析的位置的方法</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>6</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923118"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第二节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>语法文件之间的关<span lang=EN-US>系</span></span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>8</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923119"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第三节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>静态分析语法文件的方法</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>9</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923120"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第四节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>动态分析语法文件的方法</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>14</span></a></span></p>

<p class=MsoToc2><span lang=EN-US><a href="#_Toc368923121"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第五章</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>语义分析阶段</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>16</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923122"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第一节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>语义处理模块的关系</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>17</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923123"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第二节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>高端大气的模板方法模式</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>18</span></a></span></p>

<p class=MsoToc3><span lang=EN-US><a href="#_Toc368923124"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第三节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span>SemanticAnalyzer<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>的实现过程</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>20</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923125"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>1<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>处理位置别名</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>21</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923126"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>2<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>分析创建表命令</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>23</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923127"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>3<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>分析创建视图命令</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>26</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923128"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>4<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>第一阶段分析</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>26</span></a></span></p>

<p class=MsoToc5><span lang=EN-US><a href="#_Toc368923129"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>1<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小小节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>处理逻辑流程</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>26</span></a></span></p>

<p class=MsoToc5><span lang=EN-US><a href="#_Toc368923130"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>2<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小小节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>构造覆盖大多数子节点的例子</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>28</span></a></span></p>

<p class=MsoToc5><span lang=EN-US><a href="#_Toc368923131"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>3<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小小节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span>select<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>从句的处理</span></span>(TOK_SELECTDI<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>和</span></span> TOK_SELECT<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>节点</span></span>)<span
style='color:windowtext;display:none;text-decoration:none'> </span><span
style='color:windowtext;display:none;text-decoration:none'>29</span></a></span></p>

<p class=MsoToc5><span lang=EN-US><a href="#_Toc368923132"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>4<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小小节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>关于自定义函数的插曲</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>33</span></a></span></p>

<p class=MsoToc5><span lang=EN-US><a href="#_Toc368923133"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>5<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小小节</span></span><span
style='color:windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>其他的</span></span>token<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>的处理</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>33</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923134"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>5<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>产生执行计划</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>34</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923135"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>6<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>根据选项设置的优化器进行优化</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>34</span></a></span></p>

<p class=MsoToc4><span lang=EN-US><a href="#_Toc368923136"><span lang=EN-US
style='font-family:宋体'><span lang=EN-US>第</span></span>7<span lang=EN-US
style='font-family:宋体'><span lang=EN-US>小节</span></span><span style='color:
windowtext;text-decoration:none'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span
lang=EN-US style='font-family:宋体'><span lang=EN-US>产生</span></span>mr<span
lang=EN-US style='font-family:宋体'><span lang=EN-US>执行任务</span></span><span
style='color:windowtext;display:none;text-decoration:none'>... </span><span
style='color:windowtext;display:none;text-decoration:none'>34</span></a></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h2><a name="_Toc368923110"><span lang=EN-US>第一章<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>前言</span></a></h2>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>我们知道</span><span lang=EN-US
style='font-size:14.0pt'> hive</span><span style='font-size:14.0pt;font-family:
宋体'>是一个基于</span><span lang=EN-US style='font-size:14.0pt'>hadoop</span><span
style='font-size:14.0pt;font-family:宋体'>上面提供了类</span><span lang=EN-US
style='font-size:14.0pt'>sql</span><span style='font-size:14.0pt;font-family:
宋体'>的接口的组件，以方便熟悉</span><span lang=EN-US style='font-size:14.0pt'>sql</span><span
style='font-size:14.0pt;font-family:宋体'>的人使用</span><span lang=EN-US
style='font-size:14.0pt'>hadoop</span><span style='font-size:14.0pt;font-family:
宋体'>，对大数据的处理。限于精力，本文只关注</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>所提供的语法，它解析语法的过程，以及生成执行计划和优化这几个过程。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>本文可以作为</span><span lang=EN-US
style='font-size:14.0pt'>antlr</span><span style='font-size:14.0pt;font-family:
宋体'>实现的高级进阶教程，亦可以作为</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>的源码分析教程，毕竟，语法词法解析，执行计划的生成优化以及执行也是</span><span
lang=EN-US style='font-size:14.0pt'>hive</span><span style='font-size:14.0pt;
font-family:宋体'>的核心。这不是定位不清晰，而是我有这样的需求，既想了解</span><span lang=EN-US
style='font-size:14.0pt'>antlr</span><span style='font-size:14.0pt;font-family:
宋体'>的具体使用，又想深入了解</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>本文所用到的自绘的图尽量采用</span><span lang=EN-US
style='font-size:14.0pt'>graphviz</span><span style='font-size:14.0pt;
font-family:宋体'>生成，考虑到他们放入</span><span lang=EN-US style='font-size:14.0pt'>word</span><span
style='font-size:14.0pt;font-family:宋体'>会失真，所以给提供了最原始的生成文件，你可以按照自己的需要生成各种格式，他们都存放在</span><span
lang=EN-US style='font-size:14.0pt'>dot_file</span><span style='font-size:14.0pt;
font-family:宋体'>的目录下。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>最后，本文随时会根据作者的喜好和发现而做调整，如果你觉得那些地方不妥，请来信指正或者指教</span><span
lang=EN-US style='font-size:14.0pt'>:workspace.public@gmail.com</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h2><a name="_Toc368923111"><span lang=EN-US>第二章<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>准备工作</span></a></h2>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>分析对象</span><span
lang=EN-US style='font-size:14.0pt'>:hive</span><span style='font-size:14.0pt;
font-family:宋体'>源代码，版本</span><span lang=EN-US style='font-size:14.0pt'>Revision
1522497</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>需要用到的工具：</span><span
lang=EN-US style='font-size:14.0pt'> svn ant&nbsp; antlr-3.5-complete.jar</span><span
style='font-size:14.0pt;font-family:宋体'>，</span><span lang=EN-US
style='font-size:14.0pt'>antlrworks-1.4.2.jar</span><span style='font-size:
14.0pt;font-family:宋体'>，</span><span lang=EN-US style='font-size:14.0pt'>eclipse</span><span
style='font-size:14.0pt;font-family:宋体'>。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>注：在</span><span
lang=EN-US style='font-size:14.0pt'>http://www.antlr3.org/download.html&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>可以下载</span><span lang=EN-US
style='font-size:14.0pt'>antlr-3.5-complete.jar</span><span style='font-size:
14.0pt;font-family:宋体'>，</span><span lang=EN-US style='font-size:14.0pt'>antlrworks-1.4.2.jar.</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>中使用到的</span><span lang=EN-US
style='font-size:14.0pt'>antlr</span><span style='font-size:14.0pt;font-family:
宋体'>是</span><span lang=EN-US style='font-size:14.0pt'> v3.4</span><span
style='font-size:14.0pt;font-family:宋体'>版本的，使用</span><span lang=EN-US
style='font-size:14.0pt'>v3.5</span><span style='font-size:14.0pt;font-family:
宋体'>没有问题，但请别使用</span><span lang=EN-US style='font-size:14.0pt'>v4.0</span><span
style='font-size:14.0pt;font-family:宋体'>的。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>下载源代码</span><span
lang=EN-US style='font-size:14.0pt'>,</span><span style='font-size:14.0pt;
font-family:宋体'>并编译</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>svn co
http://svn.apache.org/repos/asf/hive/trunk hive</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>cd hive</span></p>

<pre style='background:white'><span lang=EN-US style='font-size:14.0pt;
font-family:"Calibri","sans-serif"'>ant clean package eclipse-files</span></pre>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>最后一个是为了方便导入成</span><span
lang=EN-US style='font-size:14.0pt'>eclipse</span><span style='font-size:14.0pt;
font-family:宋体'>项目，有些人是喜欢用</span><span lang=EN-US style='font-size:14.0pt'>eclipse</span><span
style='font-size:14.0pt;font-family:宋体'>查看源代码。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>安装部署好分布式的</span><span
lang=EN-US style='font-size:14.0pt'>hadoop</span><span style='font-size:14.0pt;
font-family:宋体'>，并安装好</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>。可以跑例子</span><span lang=EN-US
style='font-size:14.0pt'>...</span><span style='font-size:14.0pt;font-family:
宋体'>具体的可以参考网络上的教程</span><span lang=EN-US style='font-size:14.0pt'>,</span><span
style='font-size:14.0pt;font-family:宋体'>一搜一大把。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>文中准备的是官方网的测试数据。所以你需要执行以下语句</span><span
lang=EN-US style='font-size:14.0pt'>:</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>CREATE TABLE
invites (foo INT, bar STRING) PARTITIONED BY (ds STRING);</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>LOAD DATA LOCAL
INPATH '../examples/files/kv2.txt' OVERWRITE INTO TABLE invites PARTITION
(ds='2008-08-15');</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>LOAD DATA LOCAL
INPATH '../examples/files/kv3.txt' OVERWRITE INTO TABLE invites PARTITION
(ds='2008-08-08');</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>注意</span><span
lang=EN-US style='font-size:14.0pt'>:</span><span style='font-size:14.0pt;
font-family:宋体'>数据文件在本地的位置要根据你启动</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>的不同而改变。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h2><a name="_Toc368923112"><span lang=EN-US>第三章<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>总体概览</span></a></h2>

<h3><a name="_Toc368923113"><span lang=EN-US>第一节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span lang=EN-US>hive</span></a><span style='font-family:宋体'>官方架构图</span></h3>

<p class=MsoNormal><span lang=EN-US><img width=554 height=310
src="hive词法语法解析分析.files/image001.jpg"></span></p>

<p class=MsoNormal><span style='font-family:宋体'>摘自</span><span lang=EN-US>
hadoop </span><span style='font-family:宋体'>权威指南</span></p>

<p class=MsoNormal><span lang=EN-US>&nbsp;</span></p>

<h3><a name="_Toc368923114"><span lang=EN-US>第二节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>流程处理图</span></a></h3>

<p class=MsoNormal><span lang=EN-US>&nbsp; <img width=617 height=930
src="hive词法语法解析分析.files/image002.png" alt="proccess_flow.dot.png"></span></p>

<p class=MsoNormal><span style='font-family:宋体'>这张图是从源代码中剥茧抽丝的出来的抽象，忽略掉了无关紧要的细节。</span></p>

<p class=MsoNormal><span style='font-family:宋体'>这张图描述了</span><span lang=EN-US>
hive</span><span style='font-family:宋体'>从解析语句到执行的过程。</span> <span
style='font-family:宋体'>后续的内容会以为大纲详细展开。</span></p>

<h3><a name="_Toc368923115"><span lang=EN-US>第三节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>具体框架图</span></a></h3>

<p class=MsoNormal><span lang=EN-US>(</span><span style='font-family:宋体'>待完善</span><span
lang=EN-US>)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h2><a name="_Toc368923116"><span lang=EN-US>第四章<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>语法分析</span></a></h2>

<p class=MsoNormal><span lang=EN-US>&nbsp;&nbsp;&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>对一个已有的项目进行语法分析，首先是要找到语法分析文件，分析语法文件之间的关系，然后根据语法分析文件提供的接口，确定语法解析在这个体系中所处的位置。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
</span><span style='font-size:14.0pt;font-family:宋体'>在分析语法文件的时候，一般常用两种分析方法</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>静态分析和动态分析方法。静态分析方法常见是读语法文件，</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>或者使用可视化工具查看语法文件。动态分析方法是运行该解析程序，查看运行过程中语法识别的情况以及相关的具体语法树的构建过程以及抽象语法树构建过程</span><span
lang=EN-US style='font-size:14.0pt'>(</span><span style='font-size:14.0pt;
font-family:宋体'>如果可能有抽象语法树的话</span><span lang=EN-US style='font-size:14.0pt'>)</span><span
style='font-size:14.0pt;font-family:宋体'>。</span></p>

<p class=MsoNormal><span lang=EN-US>&nbsp;</span></p>

<h3><a name="_Toc368923117"><span lang=EN-US>第一节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>快速定位词法语法分析的位置的方法</span></a></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>搜索</span><span
lang=EN-US style='font-size:14.0pt'> .g</span><span style='font-size:14.0pt;
font-family:宋体'>的位置</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>find ./ -name
&quot;*.g&quot;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>会得到</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>基本可以确定了</span><span
lang=EN-US style='font-size:14.0pt'> ql</span><span style='font-size:14.0pt;
font-family:宋体'>下面就是需要找到东西。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>find . -name
&quot;*.java&quot; |xargs grep &quot;HiveLexer &quot;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>找到两处</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./build/ql/gen/antlr/gen-java/org/apache/hadoop/hive/ql/parse/HiveLexer.java:public
class HiveLexer extends Lexer {</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>./ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java:&nbsp;
public class HiveLexerX extends HiveLexer {</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>其中第一处是编译期构建的，</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>而另外一处是包装了该类。重点可以放到第二处。也即</span><span lang=EN-US style='font-size:14.0pt'>parseDriver</span><span
style='font-size:14.0pt;font-family:宋体'>类。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>对</span><span
lang=EN-US style='font-size:14.0pt'>parseDriver</span><span style='font-size:
14.0pt;font-family:宋体'>类做一个大致的分析，</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>包装了</span><span
lang=EN-US style='font-size:14.0pt'>antlr</span><span style='font-size:14.0pt;
font-family:宋体'>的</span><span lang=EN-US style='font-size:14.0pt'>antlrStringStream
</span><span style='font-size:14.0pt;font-family:宋体'>成为</span><span lang=EN-US
style='font-size:14.0pt'>antlrnocasestream</span><span style='font-size:14.0pt;
font-family:宋体'>。消除了大小写敏感。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>包装了词法解析器</span><span
style='font-size:14.0pt'> <span lang=EN-US>HiveLexer</span></span><span
style='font-size:14.0pt;font-family:宋体'>，主要是包装了出错信息。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>包装了抽象树节点，主要聚合了</span><span
lang=EN-US style='font-size:14.0pt'>ASTNodeOrigin </span><span
style='font-size:14.0pt;font-family:宋体'>以求能获得对象类型，名字，定义，别名，和定义。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>其中</span><span
lang=EN-US style='font-size:14.0pt'>parser</span><span style='font-size:14.0pt;
font-family:宋体'>方法是我们重点关注的，它调用词法解析和语法解析。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>察看其所有的调用者的关系形成的路径，找到一条通向</span><span
lang=EN-US style='font-size:14.0pt'>cli</span><span style='font-size:14.0pt;
font-family:宋体'>的包的</span><span lang=EN-US style='font-size:14.0pt'>clidriver</span><span
style='font-size:14.0pt;font-family:宋体'>类</span><span lang=EN-US
style='font-size:14.0pt'>main</span><span style='font-size:14.0pt;font-family:
宋体'>方法，（可以通过</span><span lang=EN-US style='font-size:14.0pt'>eclipse</span><span
style='font-size:14.0pt;font-family:宋体'>的北调用关系反向查看）</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>clidriver.run()
-&gt;clidriver.executeDriver()-&gt;clidriver.processLine(string,boolean)-&gt;clidriver.processCmd()-&gt;clidriver.processloaclcmd()-&gt;driver.run-&gt;driver.runInternal()-&gt;driver.compile()-&gt;parserdriver.parse</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>而我们知道这也是其中的入口之一。</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>基本上可以快速的确定其所用到的词法语法解析了。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h3><a name="_Toc368923118"><span lang=EN-US>第二节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>语法文件之间的关系</span></a></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>翻阅几个</span><span
lang=EN-US style='font-size:14.0pt'>.g</span><span style='font-size:14.0pt;
font-family:宋体'>文件可以得知</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>HiveLexer.g </span><span
style='font-size:14.0pt;font-family:宋体'>是做词法分析的，定义了所有用到的</span><span
lang=EN-US style='font-size:14.0pt'>token</span><span style='font-size:14.0pt;
font-family:宋体'>。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>HiveParser.g </span><span
style='font-size:14.0pt;font-family:宋体'>是做语法解析的。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>FromClauseParser.g&nbsp;
from</span><span style='font-size:14.0pt;font-family:宋体'>从句语法解析</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>SelectClauseParser.g&nbsp;
select </span><span style='font-size:14.0pt;font-family:宋体'>从句语法解析。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>IdentifiersParser.g&nbsp;
</span><span style='font-size:14.0pt;font-family:宋体'>自定义函数的解析，</span><span
lang=EN-US style='font-size:14.0pt'>hive</span><span style='font-size:14.0pt;
font-family:宋体'>中的自定义函数范围很广，各种内建的库函数，包括操作符之类的都被归为自定义的函数，而在语义解析的时候给以甄别。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>关系图</span><span
lang=EN-US style='font-size:14.0pt'>:</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'><img width=554
height=273 id="图片 9" src="hive词法语法解析分析.files/image003.png"
alt="gfile_relation.dot.png"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>熟悉</span><span
lang=EN-US style='font-size:14.0pt'>antlr v3</span><span style='font-size:14.0pt;
font-family:宋体'>的朋友一般会奇怪，为什么这里的词法语法怎么不是在一个文件，或者词法语法结对出现呢？</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>这里面用到了一种叫</span><span lang=EN-US style='font-size:14.0pt'>composite
grammars</span><span style='font-size:14.0pt;font-family:宋体'>的技术。</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>这种高级货是</span><span lang=EN-US style='font-size:14.0pt'>antlr v3.1</span><span
style='font-size:14.0pt;font-family:宋体'>开始引进的。</span><span style='font-size:
14.0pt'> </span><span style='font-size:14.0pt;font-family:宋体'>是为了解决把所有语法塞入到一个文件里导致编译出来的</span><span
lang=EN-US style='font-size:14.0pt'>java</span><span style='font-size:14.0pt;
font-family:宋体'>文件过大和逻辑多了之后不容易阅读的问题。它允许在逻辑上把一个大语法划分成几大块，独立实现，然后合并在一起。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>HiveParser.g </span><span
style='font-size:14.0pt;font-family:宋体'>有一行</span><span lang=EN-US
style='font-size:14.0pt'>&nbsp; import SelectClauseParser, FromClauseParser,
IdentifiersParser;&nbsp; </span><span style='font-size:14.0pt;font-family:宋体'>类似于</span><span
lang=EN-US style='font-size:14.0pt'> c</span><span style='font-size:14.0pt;
font-family:宋体'>中的</span><span lang=EN-US style='font-size:14.0pt'> include SelectClauseParser.g,
FromClauseParser.g, IdentifiersParser.g</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h3><a name="_Toc368923119"><span lang=EN-US>第三节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>静态分析语法文件的方法</span></a></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>分析一个词法语法的实现，最好的方式是跟随着语法规则走读一边逐一标注。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>当然，更快捷有效的方式是在静态分析方面使用一些可视化的工具来察看语法结构如语法图，</span></p>

<h5><span lang=EN-US>第1小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>使用</span><span lang=EN-US>antlrworks</span><span
style='font-family:宋体'>来查看语法结构</span> </h5>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>具体的使用初学者请参照</span><span
style='font-size:14.0pt'> </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>http://www.github.com/alan2lin/hand_in_hand_with_antlr</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>中的安装与使用。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>hivelexer</span><span
style='font-size:14.0pt;font-family:宋体'>不需要关注太多，就是识别关键字和</span><span
lang=EN-US style='font-size:14.0pt'>token</span><span style='font-size:14.0pt;
font-family:宋体'>的，重点是</span><span lang=EN-US style='font-size:14.0pt'>
hiveparser</span><span style='font-size:14.0pt;font-family:宋体'>。</span></p>

<h5><span lang=EN-US>第2小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span lang=EN-US>antlr</span><span style='font-family:宋体'>语法的简单介绍</span></h5>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>关于</span><span
lang=EN-US style='font-size:14.0pt'>antlr</span><span style='font-size:14.0pt;
font-family:宋体'>语法规范，详细请参见</span><span lang=EN-US style='font-size:14.0pt'>
hand in hand with antlr</span><span style='font-size:14.0pt;font-family:宋体'>中的语法规范的翻译。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>这里只做简单的介绍，以便没有基础的人也可以读懂。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>parser grammar
HiveParser;&nbsp; &nbsp;&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>标准格式</span><span style='font-size:14.0pt'>
<span lang=EN-US>parser grammar </span></span><span style='font-size:14.0pt;
font-family:宋体'>后面跟语法名。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>options //</span><span
style='font-size:14.0pt;font-family:宋体'>选项</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>{</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>tokenVocab=HiveLexer;&nbsp;
//</span><span style='font-size:14.0pt;font-family:宋体'>词汇表来源于</span><span
style='font-size:14.0pt'> <span lang=EN-US>HiveLexer</span></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>output=AST;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
//</span><span style='font-size:14.0pt;font-family:宋体'>输出</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>抽象语法树</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>ASTLabelType=CommonTree;&nbsp;
//</span><span style='font-size:14.0pt;font-family:宋体'>抽象语法树类型为</span><span
lang=EN-US style='font-size:14.0pt'> commonTtree</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>backtrack=false;&nbsp;
//</span><span style='font-size:14.0pt;font-family:宋体'>不回溯</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>k=3;&nbsp;&nbsp;
//</span><span style='font-size:14.0pt;font-family:宋体'>前向窥看</span><span
lang=EN-US style='font-size:14.0pt'>3</span><span style='font-size:14.0pt;
font-family:宋体'>个</span><span lang=EN-US style='font-size:14.0pt'>token</span><span
style='font-size:14.0pt;font-family:宋体'>的长度。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>}</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>import
SelectClauseParser, FromClauseParser, IdentifiersParser;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//map</span><span
style='font-size:14.0pt;font-family:宋体'>建立的过程</span><span lang=EN-US
style='font-size:14.0pt'>....</span><span style='font-size:14.0pt;font-family:
宋体'>省略</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>整个规则由</span><span lang=EN-US
style='font-size:14.0pt'>statement</span><span style='font-size:14.0pt;
font-family:宋体'>开始。</span><span lang=EN-US style='font-size:14.0pt'> statement </span><span
style='font-size:14.0pt;font-family:宋体'>由</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>解释语句</span><span
lang=EN-US style='font-size:14.0pt'>explainStatement</span><span
style='font-size:14.0pt;font-family:宋体'>或执行语句</span><span lang=EN-US
style='font-size:14.0pt'>execStatement</span><span style='font-size:14.0pt;
font-family:宋体'>组成。这种形式叫产生式</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>冒号左边的是左部，作为代表这个产生式规则的符号。</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>冒号右边是右部，</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>连接符号</span><span lang=EN-US
style='font-size:14.0pt'>&nbsp; | </span><span style='font-size:14.0pt;
font-family:宋体'>表示</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>右部的组成部分是或者的关系，还有另一种解读的意思是</span><span
lang=EN-US style='font-size:14.0pt'>:</span><span style='font-size:14.0pt;
font-family:宋体'>从</span><span lang=EN-US style='font-size:14.0pt'>statement</span><span
style='font-size:14.0pt;font-family:宋体'>开始，产生了</span><span lang=EN-US
style='font-size:14.0pt'> explainstatement </span><span style='font-size:14.0pt;
font-family:宋体'>或者</span><span lang=EN-US style='font-size:14.0pt'>
execstatement</span><span style='font-size:14.0pt;font-family:宋体'>。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>// starting rule</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>statement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; :
explainStatement EOF</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; |
execStatement EOF</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>解释语句</span><span style='font-size:14.0pt'>
<span lang=EN-US>explainStatement </span></span><span style='font-size:14.0pt;
font-family:宋体'>由</span><span lang=EN-US style='font-size:14.0pt'>KW_EXPLAIN </span><span
style='font-size:14.0pt;font-family:宋体'>开始</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>中间有可选项</span><span
style='font-size:14.0pt'> <span lang=EN-US>KW_EXTENDED</span></span><span
style='font-size:14.0pt;font-family:宋体'>，</span><span lang=EN-US
style='font-size:14.0pt'>KW_FORMATTED&nbsp; KW_DEPENDENCY KW_LOGICAL </span><span
style='font-size:14.0pt;font-family:宋体'>，</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>后面紧跟着</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>执行语句</span><span style='font-size:14.0pt'> </span><span style='font-size:
14.0pt;font-family:宋体'>。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>通过观测，</span><span
lang=EN-US style='font-size:14.0pt'>KW_ </span><span style='font-size:14.0pt;
font-family:宋体'>开始的</span><span lang=EN-US style='font-size:14.0pt'>token</span><span
style='font-size:14.0pt;font-family:宋体'>代表</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>关键字。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>语法形式</span><span
lang=EN-US style='font-size:14.0pt'>: </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init </span><span
style='font-size:14.0pt;font-family:宋体'>表示进入规则时执行后面的</span><span lang=EN-US
style='font-size:14.0pt'>{}</span><span style='font-size:14.0pt;font-family:
宋体'>里的动作</span><span style='font-size:14.0pt'> </span><span style='font-size:
14.0pt;font-family:宋体'>，例中，压入</span><span lang=EN-US style='font-size:14.0pt'>trace</span><span
style='font-size:14.0pt;font-family:宋体'>的消息。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after{} </span><span
style='font-size:14.0pt;font-family:宋体'>表示规则完成后执行</span><span lang=EN-US
style='font-size:14.0pt'>{}</span><span style='font-size:14.0pt;font-family:
宋体'>里面的动作。</span><span lang=EN-US style='font-size:14.0pt'>&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体'>例中，弹出</span><span lang=EN-US
style='font-size:14.0pt'>trace</span><span style='font-size:14.0pt;font-family:
宋体'>的消息。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>-&gt;</span><span
style='font-size:14.0pt;font-family:宋体'>构建语法抽象树</span><span lang=EN-US
style='font-size:14.0pt'> ^(rootnode&nbsp; leafnode1 leafnode2...) </span><span
style='font-size:14.0pt;font-family:宋体'>如例</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>表示构建一个以</span><span
style='font-size:14.0pt'> <span lang=EN-US>TOK_EXPLAIN </span></span><span
style='font-size:14.0pt;font-family:宋体'>为根节点</span><span lang=EN-US
style='font-size:14.0pt'>&nbsp;&nbsp; execStatement </span><span
style='font-size:14.0pt;font-family:宋体'>为第一个叶结点，</span><span style='font-size:
14.0pt'> </span><span style='font-size:14.0pt;font-family:宋体'>可选项为第二个叶结点，如果有可选项的话。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>explainOptions=KW_EXTENDED
</span><span style='font-size:14.0pt;font-family:宋体'>定义了</span><span
style='font-size:14.0pt'> <span lang=EN-US>explainOptions</span></span><span
style='font-size:14.0pt;font-family:宋体'>作为别名引用</span><span lang=EN-US
style='font-size:14.0pt'>KW_EXTENDED</span><span style='font-size:14.0pt;
font-family:宋体'>，</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>引用形式为</span><span lang=EN-US
style='font-size:14.0pt'> $ explainOptions.</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>explainStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init {
msgs.push(&quot;explain statement&quot;); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after {
msgs.pop(); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; :
KW_EXPLAIN (explainOptions=KW_EXTENDED|explainOptions=KW_FORMATTED|explainOptions=KW_DEPENDENCY|explainOptions=KW_LOGICAL)?
execStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-&gt; ^(TOK_EXPLAIN execStatement $explainOptions?)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; ;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>执行语句</span><span lang=EN-US
style='font-size:14.0pt'>execStatement </span><span style='font-size:14.0pt;
font-family:宋体'>由</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>查询，装载，导出，导入，数据定义</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>四大语句。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>execStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init {
msgs.push(&quot;statement&quot;); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after {
msgs.pop(); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp; &nbsp;:
queryStatementExpression</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
| loadStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
| exportStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
| importStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
| ddlStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>装载语句</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>只关注</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>路径，表或分区，</span><span style='font-size:14.0pt'> </span><span
style='font-size:14.0pt;font-family:宋体'>是否定义了本地，</span><span style='font-size:
14.0pt'> </span><span style='font-size:14.0pt;font-family:宋体'>是否定义了重写。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>loadStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init {
msgs.push(&quot;load statement&quot;); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after {
msgs.pop(); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
: KW_LOAD KW_DATA (islocal=KW_LOCAL)? KW_INPATH (path=StringLiteral)
(isoverwrite=KW_OVERWRITE)? KW_INTO KW_TABLE (tab=tableOrPartition)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
-&gt; ^(TOK_LOAD $path $tab $islocal? $isoverwrite?)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>导出语句</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>只关注</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>表或分区</span><span style='font-size:14.0pt'> </span><span style='font-size:
14.0pt;font-family:宋体'>和导出路径</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>exportStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init { msgs.push(&quot;export
statement&quot;); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after {
msgs.pop(); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
: KW_EXPORT KW_TABLE (tab=tableOrPartition) KW_TO (path=StringLiteral)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
-&gt; ^(TOK_EXPORT $tab $path)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>//</span><span
style='font-size:14.0pt;font-family:宋体'>导入语句</span><span style='font-size:14.0pt'>
</span><span style='font-size:14.0pt;font-family:宋体'>只关注导入路径，表或着分区，</span><span
style='font-size:14.0pt'> </span><span style='font-size:14.0pt;font-family:
宋体'>是否是外部</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>importStatement</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@init {
msgs.push(&quot;import statement&quot;); }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>@after { msgs.pop();
}</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; :
KW_IMPORT ((ext=KW_EXTERNAL)? KW_TABLE (tab=tableOrPartition))? KW_FROM
(path=StringLiteral) tableLocation?</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
-&gt; ^(TOK_IMPORT $path $tab? $ext? tableLocation?)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>以此类推，不再累述。</span></p>

<h3><a name="_Toc368923120"><span lang=EN-US>第四节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>动态分析语法文件的方法</span></a></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>这里介绍一种更为直观的的方法，把</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>.g</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>中的语义动作清除，然后用</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>antlrworks</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>查看，并调试。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>在</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>antlrworks</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>中能够以图形的方式展示语法结构</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>(</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>只限于一个规则一个规则的显示图</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>)</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>如</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=460 height=80 src="hive词法语法解析分析.files/image004.png"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>或者具体语法分析和生成的语法解析树，</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>例如，官网中的例子</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>SELECT
a.foo FROM invites a WHERE a.ds='2008-08-15'</span></span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>生成的具体语法树</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal align=left style='text-align:left'><span lang=EN-US
style='font-size:12.0pt;font-family:宋体'><img width=658 height=582
src="hive词法语法解析分析.files/image005.jpg"></span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>生成的抽象语法树</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal align=left style='text-align:left'><span lang=EN-US
style='font-size:12.0pt;font-family:宋体'><img width=630 height=365 id="图片 6"
src="hive词法语法解析分析.files/image006.jpg"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>要想了解语法，只能逐一解读，别无它法。而这种产生式是最简约的描述信息了，就不重复贴出来了。附件是清理后的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> .g</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>文件，可直接使用。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>好吧，如果你真的读完了，你所能得到的也只是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>hive ql</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>手册能提供的语法功能，当然，额外奖励是，你知道这些功能是如何更具体的被描述的。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h2><a name="_Toc368923121"><span lang=EN-US>第五章<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>语义分析阶段</span></a></h2>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>从</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>hive</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>官方结构图也能看出来，</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>driver</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>类是关键。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>本章节按照处理流程图中的阶段依次展开。</span></p>

<h3><a name="_Toc368923122"><span lang=EN-US>第一节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>语义处理模块的关系</span></a></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>语义模块的输入是一个抽象语法树，</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>输出也是一个抽象语法树，但是被修剪变换过的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>.</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>hive</span><span style='font-size:14.0pt;font-family:宋体;
color:black'>采用了工厂模式来实现语义模块之间的关系。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>工厂根据抽象语法树的根节点来生产具体的语义处理器。如图</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=555 height=160 src="hive词法语法解析分析.files/image007.png"
alt=semantic.dot.png></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>各个具体实现类的具体意义如表格所示</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<table class=MsoTableGrid border=1 cellspacing=0 cellpadding=0 width=603
 style='width:451.95pt;border-collapse:collapse;border:none'>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>语义分析器的实现类</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border:solid black 1.0pt;
  border-left:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>实现意义</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>ExplainSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>只分析执行计划</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>LoadSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>装载语句的语义解析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>ExportSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>导出语句的语义解析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>ImportSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>导入语句的语义解析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>DDLSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>数据定义语言的语义解析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>FunctionSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>函数的语义解析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>ColumnStatsSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>列统计语义分析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>MacroSemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>宏语义分析</span></p>
  </td>
 </tr>
 <tr>
  <td width=304 valign=top style='width:227.65pt;border:solid black 1.0pt;
  border-top:none;padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
  color:black;background:silver'>SemanticAnalyzer</span></p>
  </td>
  <td width=299 valign=top style='width:224.3pt;border-top:none;border-left:
  none;border-bottom:solid black 1.0pt;border-right:solid black 1.0pt;
  padding:0cm 5.4pt 0cm 5.4pt'>
  <p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>常用的语义分析，主要是查询。</span></p>
  </td>
 </tr>
</table>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>具体的语义处理器与根节点的操作类型的对应关系</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=554 height=298 src="hive词法语法解析分析.files/image008.png"
alt=tok2analyzer.dot.png></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>由于具体的实现非常多，所以这里只抽取了最常见的</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US
style='background:silver'>SemanticAnalyzer</span><span lang=EN-US> </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>做具体的解析。其他的具体语义分析器，有时间我再继续。</span></p>

<h3><a name="_Toc368923123"><span lang=EN-US>第二节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>高端大气的模板方法模式</span></a></h3>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; </span><span style='font-size:14.0pt;
font-family:宋体;color:black'>在</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>hive</span><span style='font-size:14.0pt;
font-family:宋体;color:black'>的源码中的，源码清晰可读多数归结于运用了模板方法模式。无论是在</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>driver</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>的</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>runinternal</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>方法中</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>还是在</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>compile</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>方法中，都可见其痕迹。使用这种方法的好处是，能够将层级的边界界定的很清晰，对于每一个层级，都抽象出顶级逻辑作为骨架，顶级逻辑层的每一步骤，皆可留待具体方法或者具体实现类去实现，这种方式也能够清晰的界定每一个逻辑步骤的边界。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; </span><span style='font-size:14.0pt;
font-family:宋体;color:black'>对待这种模式解析的最佳方法就是擒贼先擒王，找到顶级逻辑的骨架，一切即可迎刃而解。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp; </span><span style='font-size:14.0pt;font-family:宋体;
color:black'>源码中的关于可配置的钩子的实现方式，都是使用这种方法。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>例如代码</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
List&lt;HiveSemanticAnalyzerHook&gt; saHooks =</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
getHooks(HiveConf.ConfVars.</span><i><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:#0000C0'>SEMANTIC_ANALYZER_HOOK</span></i><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>,</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
HiveSemanticAnalyzerHook.</span><b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:#7F0055'>class</span></b><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas'>&nbsp;</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#3F7F5F'>// Do semantic analysis and plan generation</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>if</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> (saHooks != </span><b><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>null</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
HiveSemanticAnalyzerHookContext hookCtx = </span><b><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>new</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>
HiveSemanticAnalyzerHookContextImpl();</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
hookCtx.setConf(</span><span lang=EN-US style='font-size:14.0pt;font-family:
Consolas;color:#0000C0'>conf</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>for</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> (HiveSemanticAnalyzerHook hook : saHooks) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<span style='background:yellow'>tree</span> = hook.preAnalyze(hookCtx, <span
style='background:silver'>tree</span>);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
}</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
sem.analyze(<span style='background:silver'>tree</span>, </span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#0000C0'>ctx</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
hookCtx.update(sem);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>for</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> (HiveSemanticAnalyzerHook hook : saHooks) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
hook.postAnalyze(hookCtx, sem.getRootTasks());</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
}</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
} </span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>else</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
sem.analyze(<span style='background:silver'>tree</span>, </span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#0000C0'>ctx</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>);</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; }</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>浓缩起来</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>其实就是三个逻辑步骤</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>执行前置钩子</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>执行分析</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>执行后置钩子</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h3><a name="_Toc368923124"><span lang=EN-US>第三节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span lang=EN-US>SemanticAnalyzer</span></a><span
style='font-family:宋体'>的实现过程</span></h3>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>在</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black;background:
silver'>BaseSemanticAnalyzer</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>.<u><span style='background:silver'> analyze</span>()</u></span><u><span
style='font-size:14.0pt;font-family:宋体;color:black'>函数中，也同样的用了模板方法模式。</span></u></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp; </span><b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>public</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> </span><b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>void</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> <span
style='background:silver'>analyze</span>(ASTNode ast, Context ctx) </span><b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>throws</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>
SemanticException {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
initCtx(ctx);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
init();</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
analyzeInternal(ast);</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp; }</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>这一层的逻辑很简单，</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>initctx</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>init</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>都是基本方法，而</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>analyzeInternal
</span><span style='font-size:14.0pt;font-family:宋体;color:black'>是抽象方法，由子类实现。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>它的实现过程，也就是我们最关注的。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>参照流程处理中的图。我们对每一个阶段进行详细展开。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h4><a name="_Toc368923125"><span lang=EN-US>第1小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>处理位置别名</span></a></h4>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>定位到</span><span
lang=EN-US style='font-size:14.0pt'>TOK_SELECT </span><span style='font-size:
14.0pt;font-family:宋体'>所在的那个层级的节点。对</span><span lang=EN-US style='font-size:
14.0pt'>group by </span><span style='font-size:14.0pt;font-family:宋体'>或者</span><span
lang=EN-US style='font-size:14.0pt'> order by</span><span style='font-size:
14.0pt;font-family:宋体'>中使用到的位置别名进行替换。不断的进行这个操作。直到遍历完树。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>通过例子，我们可以清晰的看到这个处理过程。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>在</span><span
lang=EN-US style='font-size:14.0pt'>antlrwork</span><span style='font-size:
14.0pt;font-family:宋体'>中执行（请注意，</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>中在进行语法解析之前将</span><span lang=EN-US
style='font-size:14.0pt'>token</span><span style='font-size:14.0pt;font-family:
宋体'>全部转换成大写了，所以关键字要用大写，否则通不过语法校验）</span><span style='font-size:14.0pt'> </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>SELECT foo
,count(bar) FROM invites GROUP BY foo;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>和</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>SELECT foo
,count(bar) FROM invites GROUP BY 1;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>第二条语句并不能在</span><span
lang=EN-US style='font-size:14.0pt'>hive</span><span style='font-size:14.0pt;
font-family:宋体'>中执行，在生成执行计划的时候抛出了异常，这个可能是它们的一个</span><span lang=EN-US
style='font-size:14.0pt'>bug</span><span style='font-size:14.0pt;font-family:
宋体'>。我们之后如有时间再进行分析，本节的重点是揭示</span><span lang=EN-US style='font-size:14.0pt'>hive</span><span
style='font-size:14.0pt;font-family:宋体'>如何处理位置别名的。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>SELECT foo
,count(bar) FROM invites GROUP BY foo;</span><span style='font-size:14.0pt;
font-family:宋体'>的抽象语法树</span></p>

<p class=MsoNormal><span lang=EN-US><img width=553 height=276 id="图片 1"
src="hive词法语法解析分析.files/image009.jpg"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>SELECT foo
,count(bar) FROM invites GROUP BY 1;</span><span style='font-size:14.0pt;
font-family:宋体'>的抽象语法树</span></p>

<p class=MsoNormal><span lang=EN-US><img width=554 height=268 id="图片 4"
src="hive词法语法解析分析.files/image010.jpg"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black;background:silver'>processPositionAlias</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>函数遍历每一个节点的子树，在隐含着</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_SELECT,TOK_GROUPBY,TOK_ORDERBY
</span><span style='font-size:14.0pt;font-family:宋体;color:black'>必在同一个层级，以及后两者不能并存的前提下</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>,</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>如果它们有的话。对后两者存在的，以数字来标识列位置的标示符号进行替换，替换为</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_SELECT</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>所对应的真实的列，他们可能是一个计算列。替换的过程是列标号所对应位置的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_SELEEXPR</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>的子节点。上例中是</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>拷贝</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'> TOK_TABLE_OR_COL</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>以及子节点</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>foo </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>替换掉</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'> 1.</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>有兴趣的可以尝试一下</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>SELECT foo ,bar&nbsp; FROM invites order BY foo desc;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>SELECT foo ,bar&nbsp; FROM invites order BY 1 desc;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>其中第二句可以通过语法解析和语义解析，但是结果非预期，可能是其中的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>bug</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>。在后续的分析过程中将会尝试解开这个原因。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US>&nbsp;</span></p>

<h4><a name="_Toc368923126"><span lang=EN-US>第2小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>分析创建表命令</span></a></h4>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;&nbsp;&nbsp;
</span><span style='font-size:14.0pt;font-family:宋体'>如果当前的抽象语法树的节点是</span><span
style='font-size:14.0pt'> </span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>TOK_CREATETABLE</span><span style='font-size:
14.0pt;font-family:宋体;color:black'>那么将进入分析创建表命令的处理过程。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; </span><span style='font-size:14.0pt;
font-family:宋体;color:black'>有三种创建表的情况，</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>1)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>正常的</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>语句。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span><span style='font-size:14.0pt;font-family:宋体;
color:black'>例如</span><span style='font-size:14.0pt;font-family:Consolas;
color:black'> </span><span lang=EN-US style='font-size:14.0pt'>CREATE TABLE invites
(foo INT, bar STRING) PARTITIONED BY (ds STRING);</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>2)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-like </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>语句。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>例如</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> CREATE
TABLE x LIKE invites;</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>3)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-as-select
</span><span style='font-size:14.0pt;font-family:宋体;color:black'>语句。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>例如</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> CREATE
TABLE y AS SELECT * FROM invites;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>这个阶段的语义分析主要是语义检查。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>1)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>检查，</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-like </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>与</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>create-table-as-select
</span></span><span style='font-size:14.0pt;font-family:宋体;color:black'>不能共存，不能够</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>创建一个</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> table
like xx </span><span style='font-size:14.0pt;font-family:宋体;color:black'>而又</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> as select
...</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>2)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>检查</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>create-table-like
</span></span><span style='font-size:14.0pt;font-family:宋体;color:black'>与</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>create-table-as-select</span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，确定它们不能有列名。换句话说，这两种情况创建的表的模式都是拷贝过来的。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>以及语义动作</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>1)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>如果是</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'> create-table-like </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>这两种情况，它们只是影响了元数据，属于</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>ddl</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>的范围，直接交由</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> ddlwork</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>去处理，返回</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>null</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>结束处理。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>2)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>如果是</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-as-select</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>则需要获取元信息并存入</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>qb</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，返回</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>select</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>语句所在的树，表示后续还需要处理。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>3)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>填充默认的存储格式，对不同的文件格式</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，顺序文件</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> rcfile
orc</span><span style='font-size:14.0pt;font-family:宋体;color:black'>等之类的文件设置他们的输入格式类和输出格式类以及序列化反序列化类。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>4)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span
style='font-size:14.0pt;font-family:宋体;color:black'>创建表描述，存入</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>qb</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>中。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>列出这三种情况的抽象语法树，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>分析一下处理流程</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>CREATE TABLE
invites (foo INT, bar STRING) PARTITIONED BY (ds STRING)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=554 height=230 id="图片 7"
src="hive词法语法解析分析.files/image011.jpg"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>CREATE TABLE x LIKE invites</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=271 height=222 id="图片 10"
src="hive词法语法解析分析.files/image012.png"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>CREATE TABLE y AS SELECT * FROM invites</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'><img width=554
height=298 src="hive词法语法解析分析.files/image013.jpg"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体'>分</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>析建表命令处理过程会逐一处理</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_CREATETABLE</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>下面的字节点，从第二个字节点开始处理。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>如果是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>create-table
&nbsp;</span><span style='font-size:14.0pt;font-family:宋体;color:black'>，就把从</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_TABCOLLIST</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>取出列定义来创建来表描述。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>如果是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-like</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，就取出</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>like</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>表名，用来创建表描述。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>如果是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>create-table-as-select
</span><span style='font-size:14.0pt;font-family:宋体;color:black'>也是从目标表明创建表描述，同时将</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_QUERY </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>指向的节点返回。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<h4><a name="_Toc368923127"><span lang=EN-US>第3小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>分析创建视图命令</span></a></h4>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp;&nbsp; </span><span style='font-size:14.0pt;
font-family:宋体;color:black'>如果</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>TOK_CREATEVIEW </span><span style='font-size:
14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US style='font-size:
14.0pt;font-family:Consolas;color:black'>TOK_ALTERVIEW_AS </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>两个节点，则进入分析创建视图命令过程。</span></p>

<p class=MsoNormal><span lang=EN-US>&nbsp;&nbsp;&nbsp; </span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>相对于分析创建表命令而言，分析视图创建就很简单了，得到得到各列以及若干属性，用来创建视图描述，然后交由</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>ddlwork</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>处理，同时返回</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_QUERY </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>节点。本节从简。</span></p>

<h4><a name="_Toc368923128"><span lang=EN-US>第4小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>第一阶段分析</span></a></h4>

<p class=MsoNormal><span lang=EN-US>&nbsp;&nbsp; </span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;dophase1 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>比较复杂，因为涉及处理的节点类型比较多，影响的上下文也比较多。</span></p>

<h5><a name="_Toc368923129"><span lang=EN-US>第1小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>处理逻辑流程</span></a></h5>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp; </span><span style='font-size:14.0pt;font-family:
宋体;color:black'>先来看看</span><span lang=EN-US style='font-size:14.0pt;font-family:
Consolas;color:black'>dophase1</span><span style='font-size:14.0pt;font-family:
宋体;color:black'>的逻辑</span><span lang=EN-US style='font-size:14.0pt;font-family:
Consolas;color:black'>... </span><span style='font-size:14.0pt;font-family:
宋体;color:black'>我将逻辑简化掉，以便能清楚的看清这个流程。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1)</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; boolean phase1Result = true;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; boolean skipRecursion = false;</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
if (ast.getToken() != null) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
skipRecursion = true;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; switch (ast.getToken().getType()) {</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>case HiveParser.TOK_XXX: maychange(skipRecursion);</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
break;</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>default</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
skipRecursion = </span><b><span lang=EN-US style='font-size:14.0pt;font-family:
Consolas;color:#7F0055'>false</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; </span><b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>break</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>}</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>if</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> (!skipRecursion) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>int</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> child_count = ast.getChildCount();</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span><b><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:#7F0055'>for</span></b><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'> (</span><b><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:#7F0055'>int</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> child_pos
= 0; child_pos &lt; child_count &amp;&amp; phase1Result; ++child_pos) {</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
phase1Result = phase1Result &amp;&amp; <span style='background:silver'>doPhase1</span>((ASTNode)
ast.getChild(child_pos), qb, ctx_1);</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
}</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
}</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; </span><b><span lang=EN-US style='font-size:
14.0pt;font-family:Consolas;color:#7F0055;background:silver'>return</span></b><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black;background:
silver'> phase1Result;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>逻辑清晰地显示，</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> dophase1 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>采用先根的方式递归遍历抽象语法树，从左右到右边递归处理子节点。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>递归的出口是</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>在某些个节点上设置的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>skipRecursion</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，以及子节点处理的结果。</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:red'>这里面尤其要注意的是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:red'>default</span><span
style='font-size:14.0pt;font-family:宋体;color:red'>的处理，也就是，如果这个</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:red'>token </span><span
style='font-size:14.0pt;font-family:宋体;color:red'>没有匹配的，就忽略过递归处理</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>。</span></p>

<h5><a name="_Toc368923130"><span lang=EN-US>第2小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>构造覆盖大多数子节点的例子</span></a></h5>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>先构造一个覆盖比较多的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>sql</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，通过观察它生成的抽象语法树来进行解析。特别声明，这个语句没啥含义，只是为了</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>覆盖多一点的分支情况，同时是可以通过语法调试的。这个无意构造的语句居然还能够检测出一个运行时的异常。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>SELECT * FROM </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>(SELECT foo,COUNT(ISNULL(bar)) OVER(PARTITION BY foo ORDER BY
foo)&nbsp; FROM invites WHERE ds='2008-08-15'&nbsp; ) a </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>JOIN </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>(SELECT foo,COUNT(bar) FROM invites GROUP BY foo) b </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>ON a.foo = b.foo&nbsp; ORDER BY a.foo ;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>生成的抽象语法树图形如下</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=554 height=146 id="图片 13"
src="hive词法语法解析分析.files/image014.jpg"></span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>这图太小了，最好还是把语句放入</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>
antlrworks</span><span style='font-size:14.0pt;font-family:宋体;color:black'>中查看吧，可以任意放大</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>... </span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>下面就开始各个子节点的处理</span></p>

<h5><a name="_Toc368923131"><span lang=EN-US>第3小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span lang=EN-US>select</span></a><span style='font-family:宋体'>从句的处理</span><span
lang=EN-US>(TOK_SELECTDI</span><span style='font-family:宋体'>和</span> <span
lang=EN-US>TOK_SELECT</span><span style='font-family:宋体'>节点</span><span
lang=EN-US>)</span></h5>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;&nbsp;&nbsp; </span><span style='font-size:14.0pt;
font-family:宋体;color:black'>先上图，无码高清的请到</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>dot_file</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>文件夹去看</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>...</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=554 height=400 id="图片 15"
src="hive词法语法解析分析.files/image015.png" alt="dophase_select.dot.png"></span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>上图左边是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>select
selecctdi</span><span style='font-size:14.0pt;font-family:宋体;color:black'>的处理阶段的步骤，右边是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> query
block </span><span style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>queryblock
parserinfo</span><span style='font-size:14.0pt;font-family:宋体;color:black'>的内容，</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>上图清晰的展示出对</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>TOK_SELECT
TOK_SELECTDI</span><span style='font-size:14.0pt;font-family:宋体;color:black'>为根节点的语义处理，以及涉及的那些上下文的内容。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>1)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;step1 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>增加</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>select/select
distinct</span><span style='font-size:14.0pt;font-family:宋体;color:black'>的计数器</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，若有</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>hints</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，设置</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>hints</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>2)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>step2 &nbsp;</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>对</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>select</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>节点孙子节点进行遍历，递归寻找聚集函数和窗口。并对自定义函数是否要求隐含排序进行语义检查。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'><img
width=553 height=314 id="图片 16" src="hive词法语法解析分析.files/image016.jpg"></span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
style='font-size:14.0pt;font-family:宋体;color:black'>这一步主要是对</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>tok_select</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>的孙节点层进行递归遍历，处理</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>tok_function</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'> tok_functiondi </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>tok_functionstar</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>节点。如果改节点下面是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>
tok_windowspec</span><span style='font-size:14.0pt;font-family:宋体;color:black'>则认为是窗口函数，填充到出口参数，返回。如果是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>identifier
</span><span style='font-size:14.0pt;font-family:宋体;color:black'>则认为是自定义函数，</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>hive</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>中的自定义函数范围挺广的，包含常见的运算符等都是以自定义函数的方式实现的。从函数注册信息中去判断自定义函数是否需要隐含排序的条件，如果有，此时肯定缺少了</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>over</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>从句，因为</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>over</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>从句会被语法解析转换成</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>tok_windowspec</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>了</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
style='font-size:14.0pt;font-family:宋体;color:black'>语法是这样处理的，在</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>identifiersParser</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>文件中</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>//
fun(par1, par2, par3)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>function:</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
functionName</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
LPAREN</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
(</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
(star=STAR)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
| (dist=KW_DISTINCT)? (selectExpression (COMMA selectExpression)*)?</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
RPAREN (KW_OVER ws=window_specification)?</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-&gt; {$star != null}? ^(TOK_FUNCTIONSTAR functionName $ws?)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-&gt; {$dist == null}? ^(TOK_FUNCTION functionName (selectExpression+)? $ws?)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
-&gt; ^(TOK_FUNCTIONDI functionName (selectExpression+)?)</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;&nbsp;&nbsp;
;</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
style='font-size:14.0pt;font-family:宋体;color:black'>从中可以看出</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>window_specification</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>是跟在</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>KW_OVER</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>后面的，而在整个的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>.g</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>文件范围内，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>window_specification
</span></span><span style='font-size:14.0pt;font-family:宋体;color:black'>除了在定义窗口的时候，就只有在此处被引用了。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>window_specification:</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;
(Identifier | ( LPAREN Identifier? partitioningSpec? window_frame? RPAREN)) </span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>-&gt;
^(TOK_WINDOWSPEC Identifier? partitioningSpec? window_frame?);</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:28.0pt'><span
style='font-size:14.0pt;font-family:宋体;color:black'>可以清楚的看出若是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>over</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>从句，</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>tok_function </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>子节点最右边必定是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>
tok_windowspec</span><span style='font-size:14.0pt;font-family:宋体;color:black'>，而如果是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>tok_windowspec</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>在前面就被返回去了，走不到这里。故出现在此，而函数注册信息中要求隐含排序，也即是要求窗口的，而窗口是在</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>over</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>从句中出现的，此刻可以判断为缺少</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>over</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>从句。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
style='font-size:14.0pt;font-family:宋体;color:black'>语义检查中很多都是使用这样的方式，把具体语法的多种表现形式尽可能变换成统一的一致的抽象语法树形式，比如</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>加减乘除四种二元操作，变换成（操作数</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>1</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，操作符号，操作数</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>2</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>）这种抽象形式，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>当然抽象树表现的方式是</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'> ^(</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>操作符号</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp; </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>操作数</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>1 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>操作数</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>2)</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，把优先级变换成节点遍历的顺序。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>3)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>step3 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>如果是基于窗口的函数，则创建窗口描述或者对已有的窗口描述进行填充。同时对基于窗口的聚集函数创建窗口函数描述类，将这两者更新到</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>qb.</span><span
lang=EN-US> </span><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>destToWindowingSpec</span><span style='font-size:14.0pt;
font-family:宋体;color:black'>和</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>qbp.</span><span lang=EN-US> </span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>destToWindowingExprs</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>中。</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
style='font-size:14.0pt;font-family:宋体;color:black'>窗口相关的类结构如图</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>:</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'><img
width=554 height=243 id="图片 14" src="hive词法语法解析分析.files/image017.png"
alt=processWindowFunction.dot.png></span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:0cm'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>&nbsp;</span></p>

<p class=MsoListParagraph style='margin-left:21.0pt;text-indent:-21.0pt'><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>4)<span
style='font:7.0pt "Times New Roman"'>&nbsp; </span></span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>step4</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>step5 </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>只是简单的设置，代码也很简单，就不累述了。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>在这几步的处理过程中需要注意的是，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>dophase1</span><span style='font-size:14.0pt;font-family:宋体;
color:black'>的函数参数中有一个特殊的上下文</span><span style='font-size:14.0pt;font-family:
Consolas;color:black'> </span><span style='font-size:14.0pt;font-family:宋体;
color:black'>专门为该函数准备的</span></p>

<p class=MsoNormal align=left style='text-align:left;text-autospace:none'><span
style='font-size:14.0pt;font-family:宋体;color:black'>叫</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>Phase1Ctx{ String dest;
int nextNum;} dest</span><span style='font-size:14.0pt;font-family:宋体;
color:black'>在初始化的时候是</span><span lang=EN-US style='font-size:14.0pt;
font-family:Consolas;color:black'>&quot;reduce&quot; </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>只有在</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> <span lang=EN-US>TOK_DESTINATION
</span></span><span style='font-size:14.0pt;font-family:宋体;color:black'>会被改变。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h5><a name="_Toc368923132"><span lang=EN-US>第4小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>关于自定义函数的插曲</span></a></h5>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>先留一个位置，回头再行展开。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h5><span lang=EN-US>第5小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span lang=EN-US>from</span><span style='font-family:宋体'>从句的处理</span></h5>

<p class=MsoNormal><span lang=EN-US>&nbsp;</span></p>

<h5><a name="_Toc368923133"><span lang=EN-US>第6小小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>其他的</span><span lang=EN-US>token</span></a><span
style='font-family:宋体'>的处理</span></h5>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>其他的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>token</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>都是更新了上下文中的</span><span
lang=EN-US style='font-size:14.0pt;font-family:Consolas;color:black'>qb </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>和</span><span lang=EN-US
style='font-size:14.0pt;font-family:Consolas;color:black'>qbp</span><span
style='font-size:14.0pt;font-family:宋体;color:black'>，偶尔做一些简单的语义校验，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>直接上图就好了，</span><span
style='font-size:14.0pt;font-family:Consolas;color:black'> </span><span
style='font-size:14.0pt;font-family:宋体;color:black'>图示比较清晰。</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'><img width=553 height=456 id="图片 20"
src="hive词法语法解析分析.files/image018.png" alt="dophase_other_tokens.dot.png"></span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<h4><a name="_Toc368923134"><span lang=EN-US>第5小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>产生执行计划</span></a></h4>

<h4><a name="_Toc368923135"><span lang=EN-US>第6小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>根据选项设置的优化器进行优化</span></a></h4>

<h4><a name="_Toc368923136"><span lang=EN-US>第7小节<span style='font:7.0pt "Times New Roman"'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</span></span><span style='font-family:宋体'>产生</span><span lang=EN-US>mr</span></a><span
style='font-family:宋体'>执行任务</span></h4>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

<p class=MsoNormal><span style='font-size:14.0pt;font-family:宋体;color:black'>参考的引用</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>https://cwiki.apache.org/confluence/display/Hive/GettingStarted</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>https://cwiki.apache.org/confluence/display/Hive/GettingStarted+EclipseSetup</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt'>&nbsp;</span></p>

<p class=MsoNormal><span lang=EN-US style='font-size:14.0pt;font-family:Consolas;
color:black'>&nbsp;</span></p>

</div>

</body>

</html>
